package com.kwum.uctoutiao;

import java.io.FileOutputStream;
import java.io.IOException;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.kwum.TimeUtils;
import com.lowagie.text.Font;
import com.lowagie.text.Image;
import com.lowagie.text.PageSize;
import com.lowagie.text.Paragraph;
import com.lowagie.text.Rectangle;
import com.lowagie.text.rtf.RtfWriter2;

import net.sf.json.JSONArray;
import net.sf.json.JSONObject;

/**
 * @Description: 
 * @Company: yimo
 * @author: Kwum
 * @date 2018年2月22日 上午11:36:50
 */

public class Spirder {
	
	public static final String UC_CONTENT_CAPTCHA = "var xissJsonData = (.*?);";
	public static final String TENCENT_CONTENT_CAPTCHA = "<script>window.__initData = (.*?);</script>";
	public static final String P_TAG_CAPTCHA = "<p>(.*?)</p>";
	
	public static String uc_url ="http://a.mp.uc.cn/article.html?uc_param_str=frdnsnpfvecpntnwprdssskt&zzd_from=ucnews-iflow&dl_type=2&app=ucnews-iflow#!wm_aid=1be8154efae57a6194e7cab30f84310d!!wm_id=d36e444a77414f96979ddb90d38d367c";
	public static String tencent_url = "https://view.inews.qq.com/a/20180222A0VSRN00";
	public static String kuaibao_url = "https://kuaibao.qq.com/s/20180227A05XNI00?refer=kb_news&coral_uin=ec1994f6b082f10f7019e2ff9bb0d8a0d9";
	
	public static void main(String[] args) throws Exception {
		System.out.println("Go");
		
//		spirdeUCTouTiaoArticleListByTheme();
//		spirdeUCTouTiaoInfoByUrl(uc_url);
//		spirdeTencentNewsArticleListByThreme();
//		spirdeTencentNewInfoByUrl(tencent_url);
//		createWord("tx_qwe", spirdeTencentNewInfoByUrl(tencent_url));
//		createWord("uc_qwe", spirdeUCTouTiaoInfoByUrl(uc_url));
		spirdeKuaiBaoNewInfoByUrl(kuaibao_url);
//		createWord("kuaibao_qwe", spirdeKuaiBaoNewInfoByUrl(kuaibao_url));
		
		System.out.println("finish");
	}
	
	public static void spirdeTencentNewsArticleListByThreme() throws Exception {
		JSONArray array = new JSONArray();
    	JSONObject json = new JSONObject();
    	//TODO
    	json.put("name", "宠物");
    	json.put("value", "news_news_pet");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "历史");
    	json.put("value", "news_news_history");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "音乐");
    	json.put("value", "news_news_music");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "传媒");
    	json.put("value", "news_news_media");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "科普");
    	json.put("value", "news_news_kepu");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "足球");
    	json.put("value", "news_news_football");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "CBA");
    	json.put("value", "news_news_cba");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "必读");
    	json.put("value", "news_news_twentyf");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "校园");
    	json.put("value", "news_news_schoolyard");
    	array.add(json);
    	json.put("name", "要闻");
    	json.put("value", "news_video_top");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "娱乐");
    	json.put("value", "news_news_ent");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "体育");
    	json.put("value", "news_news_sports");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "军事");
    	json.put("value", "news_news_mil");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "NBA");
    	json.put("value", "news_news_nba");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "国际");
    	json.put("value", "news_news_world");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "科技");
    	json.put("value", "news_news_tech");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "财经");
    	json.put("value", "news_news_finance");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "时尚");
    	json.put("value", "news_news_lad");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "电影");
    	json.put("value", "news_news_movie");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "独家");
    	json.put("value", "news_news_orignal");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "游戏");
    	json.put("value", "news_news_game");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "房产");
    	json.put("value", "news_news_house");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "政务");
    	json.put("value", "news_news_msh");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "育儿");
    	json.put("value", "news_news_baby");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "文化");
    	json.put("value", "news_news_cul");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "佛学");
    	json.put("value", "news_news_fx");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "股票");
    	json.put("value", "news_news_istock");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "理财");
    	json.put("value", "news_news_lic");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "家居");
    	json.put("value", "news_news_jiaju");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "电竞");
    	json.put("value", "news_news_esport");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "数码");
    	json.put("value", "news_news_digi");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "星座");
    	json.put("value", "news_news_astro");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "教育");
    	json.put("value", "news_news_edu");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "美容");
    	json.put("value", "news_news_meirong");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "旅游");
    	json.put("value", "news_news_visit");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "电视剧");
    	json.put("value", "news_news_tv");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "搏击");
    	json.put("value", "news_news_strike");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "健康");
    	json.put("value", "news_news_health");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "生活");
    	json.put("value", "news_news_lifes");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "韩流");
    	json.put("value", "news_news_hanliu");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "综艺");
    	json.put("value", "news_news_zongyi");
    	array.add(json);
    	json = new JSONObject();
    	json.put("name", "美食");
    	json.put("value", "news_news_food");
    	array.add(json);
    	json = new JSONObject();
    	//TODO
    	for(int j = 0; j < array.size(); j++) {
    		long lastTime = TimeUtils.string2Timestamp(TimeUtils.getNow());
    		String lastId = "";
    		int page = 0;
    		int forward = 2;
    		String chlid = array.getJSONObject(j).getString("value");
    		String category = "";
    		int blankCount = 0;
    		int spirderCount = 0;
    		
    		switch (chlid) {
    		case "news_video_top":
    			category = "要闻";
    			break;
    		case "news_news_ent":
    			category = "娱乐";
    			break;
    		case "news_news_sports":
    			category = "体育";
    			break;
    		case "news_news_mil":
    			category = "军事";
    			break;
    		case "news_news_nba":
    			category = "NBA";
    			break;
    		case "news_news_world":
    			category = "国际";
    			break;
    		case "news_news_tech":
    			category = "科技";
    			break;
    		case "news_news_finance":
    			category = "财经";
    			break;
    		case "news_news_car":
    			category = "汽车";
    			break;
    		case "news_news_lad":
    			category = "时尚";
    			break;
    		case "news_news_movie":
    			category = "电影";
    			break;
    		case "news_news_orignal":
    			category = "独家";
    			break;
    		case "news_news_game":
    			category = "游戏";
    			break;
    		case "news_news_house":
    			category = "房产";
    			break;
    		case "news_news_msh":
    			category = "政务";
    			break;
    		case "news_video_main":
    			category = "图片";
    			break;
    		case "news_news_baby":
    			category = "育儿";
    			break;
    		case "news_news_cul":
    			category = "文化";
    			break;
    		case "news_news_fx":
    			category = "佛学";
    			break;
    		case "news_news_istock":
    			category = "股票";
    			break;
    		case "news_news_lic":
    			category = "理财";
    			break;
    		case "news_news_jiaju":
    			category = "家居";
    			break;
    		case "news_news_esport":
    			category = "电竞";
    			break;
    		case "news_news_digi":
    			category = "数码";
    			break;
    		case "news_news_astro":
    			category = "星座";
    			break;
    		case "news_news_edu":
    			category = "教育";
    			break;
    		case "news_news_meirong":
    			category = "美容";
    			break;
    		case "news_news_visit":
    			category = "旅游";
    			break;
    		case "news_news_tv":
    			category = "电视剧";
    			break;
    		case "news_news_strike":
    			category = "搏击";
    			break;
    		case "news_news_health":
    			category = "健康";
    			break;
    		case "news_news_lifes":
    			category = "生活";
    			break;
    		case "news_news_hanliu":
    			category = "韩流";
    			break;
    		case "news_news_zongyi":
    			category = "综艺";
    			break;
    		case "news_news_food":
    			category = "美食";
    			break;
    		case "news_news_pet":
    			category = "宠物";
    			break;
    		case "news_news_history":
    			category = "历史";
    			break;
    		case "news_news_music":
    			category = "音乐";
    			break;
    		case "news_news_media":
    			category = "传媒";
    			break;
    		case "news_news_kepu":
    			category = "科普";
    			break;
    		case "news_news_football":
    			category = "足球";
    			break;
    		case "news_news_cba":
    			category = "CBA";
    			break;
    		case "news_news_twentyf":
    			category = "必读";
    			break;
    		case "news_news_schoolyard":
    			category = "校园";
    			break;
    		default:
    			break;
    		}
    		
    		Document doc = Jsoup.connect("https://r.inews.qq.com/getQQNewsUnreadList?rtAd=1&needSpreadAds=1&lc_ids=&"
    				+ "forward=" + forward
    				+ "&page=" + page
    				+ "&last_id&"
    				+ "user_chlid=news_news_19,news_news_ent,news_news_sports,news_news_mil,news_news_nba,news_news_world,news_news_tech&"
    				+ "last_time=" + lastTime
    				+ "&chlid=" + chlid
    				+ "&Cookie=lskey%3D;luin%3D;skey%3D;uin%3D;%20logintype%3D0;%20main_login%3D;%20&omgid=&uid=00e08a6807b24642&"
    				+ "devid=863064010002246&appver=19_android_5.5.40&qn-rid=d6d98c1d-f70f-4d01-bef6-add818557884&"
    				+ "qn-sig=e2a0d08ee943bba66185bddf7c2e81d2")
    				.ignoreContentType(true)
    				.post();
    		System.out.println(doc.text());
    		JSONObject htmlJson = JSONObject.fromObject(doc.text());
    		
    		//获取json内容
    		JSONArray articleArray = htmlJson.getJSONArray("newslist");
    		for(int i = 0; i < articleArray.size(); i++) {
    			JSONObject artcleJson = articleArray.getJSONObject(i);
    			
    			lastId = artcleJson.optString("id");
    			forward--;
    			page++;
    			
    			TencentNew tencentNew = new TencentNew(artcleJson.optString("id"), artcleJson.optString("title"), artcleJson.optString("url"), 
    					artcleJson.optString("source"), artcleJson.optString("time"), artcleJson.optInt("comments"), category);
    			MySQLUtils.insertTencentNewList(tencentNew);
    		}
    		
    		while(spirderCount < 15) {
    			spirderCount++;
    			//每次刷新或者翻页等待3秒钟
    			Thread.sleep( 3 * 1000 );
    			
    			//第二次翻页
    			doc = Jsoup.connect("https://r.inews.qq.com/getQQNewsUnreadList?rtAd=1&lc_ids=&"
    					+ "forward=" + forward
    					+ "&page=" + page
    					+ "&last_id=" + lastId
    					+ "&newsTopPage=0&picType=1,2,2,2,2,2,1,2,1,2,2,1,2,2,2,2,1,2,1,0"
    					+ "&user_chlid=news_news_19,news_news_ent,news_news_sports,news_news_mil,news_news_nba,news_news_world,news_news_tech"
    					+ "&last_time=" + lastTime
    					+ "&chlid=" + chlid
    					+ "&Cookie=lskey%3D;luin%3D;skey%3D;uin%3D;%20logintype%3D0;%20main_login%3D;%20&omgid=&uid=00e08a6807b24642"
    					+ "&devid=863064010002246&appver=19_android_5.5.40&qn-rid=b50f9e5b-0b61-4a10-8a8f-e70416a33aa0"
    					+ "&qn-sig=8968364f52dc79a1552498050295a4e4")
    					.ignoreContentType(true)
    					.get();
    			
    			System.out.println(spirderCount + blankCount + chlid + "..." + (forward == 0 ? "刷新" : "翻页" ) + "..." + doc.text());
    			lastTime = TimeUtils.string2Timestamp(TimeUtils.getNow());
    			htmlJson = JSONObject.fromObject(doc.text());
    			articleArray = htmlJson.getJSONArray("newslist");
    			
    			//要是翻页已经拿不到数据，则进行刷新；要是刷新拿不到数据，则进行翻页
    			if(articleArray.size() == 0) {
    				if(forward == 0) {
    					forward = 1;
    					page = 0;
    					blankCount++;
    				}else {
    					forward = 0;
    					page = 0;
    					blankCount++;
    				}
    			}
    			for(int i = 0; i < articleArray.size(); i++) {
    				JSONObject artcleJson = articleArray.getJSONObject(i);
    				
    				lastId = artcleJson.optString("id");
    				page++;
    				
    				TencentNew tencentNew = new TencentNew(artcleJson.optString("id"), artcleJson.optString("title"), artcleJson.optString("url"), 
    						artcleJson.optString("source"), artcleJson.optString("time"), artcleJson.optInt("comments"), category);
    				MySQLUtils.insertTencentNewList(tencentNew);
    			}
    		}
    	}
	}
	
	/**
	 * 根据主题爬取UC头条文章列表
	 * @throws Exception
	 * @author Kwum
	 */
	public static void spirdeUCTouTiaoArticleListByTheme() throws Exception {
		JSONArray array = new JSONArray();
		JSONObject json = new JSONObject();
		//TODO
		json.put("name", "美容瘦身");
		json.put("value", "90005");
		array.add(json);
		json = new JSONObject();
		json.put("name", "时尚");
		json.put("value", "1213442674");
		array.add(json);
		json = new JSONObject();
		json.put("name", "教育");
		json.put("value", "681723207");
		array.add(json);
		json = new JSONObject();
		json.put("name", "体育");
		json.put("value", "923258246");
		array.add(json);
		json = new JSONObject();
		json.put("name", "美食");
		json.put("value", "10000");
		array.add(json);
		json = new JSONObject();
		json.put("name", "娱乐");
		json.put("value", "179223212");
		array.add(json);
		json = new JSONObject();
		json.put("name", "财经");
		json.put("value", "26325229");
		array.add(json);
		json = new JSONObject();
		json.put("name", "游戏");
		json.put("value", "169476544");
		array.add(json);
		json = new JSONObject();
		json.put("name", "育儿");
		json.put("value", "408250330");
		array.add(json);
		json = new JSONObject();
		json.put("name", "新时代");
		json.put("value", "10308");
		array.add(json);
		json = new JSONObject();
		json.put("name", "科技");
		json.put("value", "1525483516");
		array.add(json);
		json = new JSONObject();
		json.put("name", "汽车");
		json.put("value", "323644874");
		array.add(json);
		json = new JSONObject();
		json.put("name", "军事");
		json.put("value", "1105405272");
		array.add(json);json.put("name", "历史");
		json.put("value", "701104723");
		array.add(json);
		json = new JSONObject();
		json.put("name", "图片");
		json.put("value", "1964289243");
		array.add(json);
		json = new JSONObject();
		json.put("name", "涨知识");
		json.put("value", "1911322354");
		array.add(json);
		json = new JSONObject();
		json.put("name", "动漫");
		json.put("value", "10012");
		array.add(json);
		json = new JSONObject();
		json.put("name", "NBA");
		json.put("value", "90002");
		array.add(json);
		json = new JSONObject();
		json.put("name", "国际");
		json.put("value", "1001932710");
		array.add(json);
		json = new JSONObject();
		json.put("name", "股票");
		json.put("value", "90001");
		array.add(json);
		json = new JSONObject();
		json.put("name", "商业");
		json.put("value", "696724");
		array.add(json);
		json = new JSONObject();
		json.put("name", "国内足球");
		json.put("value", "684625112");
		array.add(json);
		json = new JSONObject();
		json.put("name", "国际足球");
		json.put("value", "701538712");
		array.add(json);
		json = new JSONObject();
		json.put("name", "房产");
		json.put("value", "586710362");
		array.add(json);
		json = new JSONObject();
		json.put("name", "CBA");
		json.put("value", "66498");
		array.add(json);
		json = new JSONObject();
		json.put("name", "电影");
		json.put("value", "1404457531635");
		array.add(json);
		json = new JSONObject();
		json.put("name", "互联网");
		json.put("value", "242677432");
		array.add(json);
		json = new JSONObject();
		json.put("name", "旅游");
		json.put("value", "1972619079");
		array.add(json);
		json = new JSONObject();
		json.put("name", "数码");
		json.put("value", "835729");
		array.add(json);
		json = new JSONObject();
		json.put("name", "健康");
		json.put("value", "472933935");
		array.add(json);
		json = new JSONObject();
		//TODO
    	for(int j = 0; j < array.size(); j++) {
			//初始化参数
			String channel = array.getJSONObject(j).getString("value");
			String recordId = "";
			String method = "his";
			long grabTime = 0;
			String category = "";
			int blankCount = 0;
			
			
			//确定类别，用于插入数据库
			switch (channel) {
			case "10000":
				category = "美食";
				break;
			case "169476544":
				category = "游戏";
				break;
			case "408250330":
				category = "育儿";
				break;
			case "10308":
				category = "新时代";
				break;
			case "179223212":
				category = "娱乐";
				break;
			case "923258246":
				category = "体育";
				break;
			case "26325229":
				category = "财经";
				break;
			case "1525483516":
				category = "科技";
				break;
			case "323644874":
				category = "汽车";
				break;
			case "1105405272":
				category = "军事";
				break;
			case "701104723":
				category = "历史";
				break;
			case "1964289243":
				category = "图片";
				break;
			case "1911322354":
				category = "涨知识";
				break;
			case "10012":
				category = "动漫";
				break;
			case "90002":
				category = "NBA";
				break;
			case "1001932710":
				category = "国际";
				break;
			case "90001":
				category = "股票";
				break;
			case "696724":
				category = "商业";
				break;
			case "684625112":
				category = "国内足球";
				break;
			case "701538712":
				category = "国际足球";
				break;
			case "586710362":
				category = "房产";
				break;
			case "66498":
				category = "CBA";
				break;
			case "1404457531635":
				category = "电影";
				break;
			case "242677432":
				category = "互联网";
				break;
			case "1972619079":
				category = "旅游";
				break;
			case "835729":
				category = "数码";
				break;
			case "472933935":
				category = "健康";
				break;
			case "90003":
				category = "家居";
				break;
			case "10005":
				category = "文玩";
				break;
			case "10008":
				category = "星座";
				break;
			case "674534":
				category = "健身";
				break;
			case "90005":
				category = "美容瘦身";
				break;
			case "1213442674":
				category = "时尚";
				break;
			case "681723207":
				category = "教育";
				break;
			case "90004":
				category = "留学";
				break;
			default:
				break;
			}
			
			//第一次进入
			Document document = Jsoup.connect("http://iflow.uczzd.cn/iflow/api/v1/channel/"
					+ channel
					+ "?app=ucnews-iflow&uc_param_str=dnnivebichfrmintcpgieiwidsudsvme&dn=13059059726-b309827c&nn=Kvm2jz1%2FXXcpm7TYZ6UM6vA7ZslczrXpUtrQGc6nCGo5Sw%3D%3D&ve=1.9.1.910&bi=35581&ch=tt_2&fr=android&mi=HUAWEI+MLA-AL10&nt=2&pc=&gp=KvnPyHMEFgSqPy%2Bx5OVjSvfE&wf=&ut=KvmRnLgvRwhOHfnfRd9GYClQ2pPiuHleq8wUrMTEhkYsHg%3D%3D&ai=KvlF9zTMf3cEWPuZGaozskoBrEgPWCy%2BF7m5skOeSY3z%2Fw%3D%3D&sv=hotfix&me=Kvlepv0ReR3jJpHqGIbXiB9E&content_ratio=0&ssign=KvjONmmMmrRErzVlNoMvvuEZ56%2FM1OGzEGFA6Q%2FbBEuvvWTD36ABvhcZKqVy0OHvHL4%3D&user_tag=bTkwBMD9uFfoOBSHZg%3D%3D&no_op=0&"
					+ "_tm=" + TimeUtils.string2Timestamp(TimeUtils.getNow())
					+ "&method=new&auto=1&count=0&recoid=&ftime=&puser=1")
					.ignoreContentType(true)
					.get();
			JSONObject htmlJson = JSONObject.fromObject(document.text());
			
			//获取json内容
			JSONArray articleArray = htmlJson.getJSONObject("data").getJSONArray("items");
			System.out.println(document.text());
			for(int i = 0; i < articleArray.size(); i++) {
				if(StringUtils.equalsIgnoreCase("articles", articleArray.getJSONObject(i).getString("map"))) {
					String itemId = articleArray.getJSONObject(i).getString("id");
	//				System.out.println(itemId);
					JSONObject articleJson = htmlJson.getJSONObject("data").getJSONObject("articles").getJSONObject(itemId);
					
					//获取参数以便后续翻页或者刷新会用到
					recordId = articleJson.getString("recoid");
					grabTime = articleJson.getLong("grab_time");
					
					//判断文章才插入
					if(StringUtils.startsWith(articleJson.optString("url", ""), "http://m.uczzd.cn/")) {
						//构建对象插入数据库
						UCToutiao ucToutiao = new UCToutiao(articleJson.optString("id", ""), articleJson.optString("title", ""), 
								TimeUtils.timestamp2String(articleJson.optLong("publish_time", 0)/1000), articleJson.optString("url", ""), 
								articleJson.optString("source_name", ""), articleJson.optString("origin_src_name", ""), 
								articleJson.optString("original_url", ""), category, articleJson.optInt("cmt_cnt", 0));
	//					System.out.println(ucToutiao);
						MySQLUtils.insertUCToutiaoList(ucToutiao);
					}
				}
			}
			
			while(blankCount < 10) {
				//每次刷新或者翻页等待3秒钟
				Thread.sleep( 3 * 1000 );
				
				//第二次翻页
				document = Jsoup.connect("http://iflow.uczzd.cn/iflow/api/v1/channel/"
						+ channel
						+ "?app=ucnews-iflow&uc_param_str=dnnivebichfrmintcpgieiwidsudsvme&dn=13059059726-b309827c&nn=Kvm2jz1%2FXXcpm7TYZ6UM6vA7ZslczrXpUtrQGc6nCGo5Sw%3D%3D&ve=1.9.1.910&bi=35581&ch=tt_2&fr=android&mi=HUAWEI+MLA-AL10&nt=2&pc=&gp=KvnPyHMEFgSqPy%2Bx5OVjSvfE&wf=&ut=KvmRnLgvRwhOHfnfRd9GYClQ2pPiuHleq8wUrMTEhkYsHg%3D%3D&ai=KvlF9zTMf3cEWPuZGaozskoBrEgPWCy%2BF7m5skOeSY3z%2Fw%3D%3D&sv=hotfix&me=Kvlepv0ReR3jJpHqGIbXiB9E&content_ratio=0&ssign=KvjONmmMmrRErzVlNoMvvuEZ56%2FM1OGzEGFA6Q%2FbBEuvvWTD36ABvhcZKqVy0OHvHL4%3D&user_tag=bTkwBMD9uFfoOBSHZg%3D%3D&no_op=0&"
						+ "_tm=" + TimeUtils.string2Timestamp(TimeUtils.getNow())
						+ "&method=" + method
						+ "&auto=0&count=0&recoid=" + recordId
						+ "&ftime=" + grabTime
						+ "&puser=1")
						.ignoreContentType(true)
						.get();
				try {
					htmlJson = JSONObject.fromObject(document.text());
					articleArray = htmlJson.getJSONObject("data").getJSONArray("items");
					System.out.println(blankCount + channel + "..." + (method == "new" ? "刷新" : "翻页" ) + "..." + document.text());
					
					//要是翻页已经拿不到数据，则进行刷新；要是刷新拿不到数据，则进行翻页
					if(articleArray.size() == 0) {
						if(method == "new") {
							method = "his";
							blankCount++;
						}else {
							method = "new";
							blankCount++;
						}
					}
					for(int i = 0; i < articleArray.size(); i++) {
						if(StringUtils.equalsIgnoreCase("articles", articleArray.getJSONObject(i).getString("map"))) {
							String itemId = articleArray.getJSONObject(i).getString("id");
		//					System.out.println(itemId);
							JSONObject articleJson = htmlJson.getJSONObject("data").getJSONObject("articles").getJSONObject(itemId);
							
							//获取参数以便后续翻页或者刷新会用到
							recordId = articleJson.getString("recoid");
							grabTime = articleJson.getLong("grab_time");
							
							//判断文章才插入
								if(StringUtils.startsWith(articleJson.optString("url", ""), "http://m.uczzd.cn/")) {
								//构建对象插入数据库
								UCToutiao ucToutiao = new UCToutiao(articleJson.optString("id", ""), articleJson.optString("title", ""), 
										TimeUtils.timestamp2String(articleJson.optLong("publish_time", 0)/1000), articleJson.optString("url", ""), 
										articleJson.optString("source_name", ""), articleJson.optString("origin_src_name", ""), 
										articleJson.optString("original_url", ""), category, articleJson.optInt("cmt_cnt", 0));
			//					System.out.println(ucToutiao);
								MySQLUtils.insertUCToutiaoList(ucToutiao);
							}
						}
					}
				} catch (Exception e) {
					e.printStackTrace();
				}
			}
    	}
	}
	
	/**
	 * 通过URL获取UC头条文章详情
	 * @param articleUrl 文章URL
	 * @throws IOException
	 * @author Kwum
	 */
	public static JSONArray spirdeUCTouTiaoInfoByUrl(String articleUrl) throws IOException {
		
		JSONArray result = new JSONArray();
		
		//通过jsoup获取内容
		Document document = Jsoup.connect(articleUrl)
							.ignoreContentType(true)
							.get();
		JSONObject contentJson = null;
		
		//用正则表达式过滤内容json
		Pattern p = Pattern.compile(UC_CONTENT_CAPTCHA);
        Matcher macher = p.matcher(document.body().toString());
        while (macher.find()) {
        	contentJson = JSONObject.fromObject(macher.group(1));
            break;
        }
        System.out.println(document.body().toString());
        
        //获取内容
        String content = contentJson.getString("content");
		
		//将图片分别插入内容当中
		JSONArray imgArray = contentJson.getJSONArray("images");
		for(int i = 0; i < imgArray.size(); i++) {
			JSONObject imgJson = imgArray.getJSONObject(i);
//			String imgDesc = imgJson.optString("description", "");
			String imgUrl = imgJson.optString("url", "");
			
			content = content.replace("<!--{img:" + i + "}-->", "<p><img>" + imgUrl + "</p>");
		}
		
		//通过p标签查找内容
		p = Pattern.compile(P_TAG_CAPTCHA);
        macher = p.matcher(content);
        while (macher.find()) {
        	JSONObject json = new JSONObject();
        	String con = macher.group(1);
        	if(StringUtils.startsWithIgnoreCase(con, "<img>")) {
				
				//图片
				json.put("img", con.substring(5));
			}else if(StringUtils.startsWithIgnoreCase(con, "<strong>")) {
				
				//粗体
				json.put("bold", con.substring(8, con.length()-9));
			}else {
				
				//正常字体
				json.put("normal", con);
			}
			
			//放进结果集
			result.add(json);
        }
        
        return result;
	}
	
	/**
	 * 通过URL获取腾讯新闻文章详情
	 * @param articleUrl 文章URL
	 * @throws IOException
	 * @author Kwum
	 */
	public static JSONArray spirdeTencentNewInfoByUrl(String articleUrl) throws IOException {
		
		JSONArray result = new JSONArray();
		
		//通过jsoup获取内容
		Document document = Jsoup.connect(articleUrl)
							.ignoreContentType(true)
							.get();
		JSONObject contentJson = null;
		
		//用正则表达式过滤内容json
		Pattern p = Pattern.compile(TENCENT_CONTENT_CAPTCHA);
        Matcher macher = p.matcher(document.body().toString());
        while (macher.find()) {
        	contentJson = JSONObject.fromObject(macher.group(1));
            break;
        }
        
		//处理内容
		JSONArray imgArray = contentJson.getJSONObject("content").getJSONArray("content");
		for(int i = 0; i < imgArray.size(); i++) {
			JSONObject imgJson = imgArray.getJSONObject(i);
			String imgUrl = imgJson.optString("img_url", "");
			String desc = imgJson.optString("desc", "");
			System.out.println(imgUrl + desc);
			
			JSONObject json = new JSONObject();
			if(!StringUtils.equalsIgnoreCase("", imgUrl)) {
				
				//图片
				json.put("img", imgUrl);
			}else if(!StringUtils.equalsIgnoreCase("", desc)) {
				
				//粗体
				if(StringUtils.startsWithIgnoreCase(desc, "<strong>")) {
					json.put("bold", desc.substring(8, desc.length()-9));
				}else {
					//正常字体
					json.put("normal", desc);
				}
			}
			
			//放进结果集
			result.add(json);
		}
		return result;
	}
	
	/**
	 * 通过URL获取天天快报文章详情
	 * @param articleUrl 文章URL
	 * @throws IOException
	 * @author Kwum
	 */
	public static JSONArray spirdeKuaiBaoNewInfoByUrl(String articleUrl) throws IOException {
		JSONArray result = new JSONArray();
		
		//通过jsoup获取内容
		Document document = Jsoup.connect(articleUrl)
							.ignoreContentType(true)
							.get();
		
		//获取p标签内容
		Elements elements = document.getElementsByTag("p");
		
		//处理内容
		for(Element e : elements) {
			JSONObject json = new JSONObject();
			if(StringUtils.startsWith(e.html(), "<img src")) {
				
				//图片
				json.put("img", e.html().substring(10, e.html().length()-38));
			}else if(StringUtils.startsWith(e.html(), "<strong><img src")) {
				
				//图片
				json.put("img", e.html().substring(18, e.html().length()-47));
			}else if(StringUtils.startsWith(e.html(), "<strong>")) {
				
				//粗体字体
				json.put("bold", e.html().substring(8, e.html().length()-9));
			}else if(StringUtils.startsWith(e.html(), "正在加载...")) {
			}else if(StringUtils.equals(e.html(), "")) {
			}else{
				
				//正常字体
				json.put("normal", e.html());
			}
			
			//放进结果集
			result.add(json);
		}
		return result;
	}
	
	/**
	 * 创建word文档
	 * @param title 标题
	 * @param jsonArray json数组，识别：img图片、bold粗体字、normal正常字
	 * @author Kwum
	 */
	public static void createWord(String title, JSONArray jsonArray) {
		try {  
            /** 创建Document对象(word文档) **/  
            Rectangle rectPageSize = new Rectangle(PageSize.A4);  
            rectPageSize = rectPageSize.rotate();  
              
            // 创建word文档,并设置纸张的大小  
            com.lowagie.text.Document doc = new com.lowagie.text.Document(PageSize.A4);  
              
            //建立一个书写器与document对象关联,通过书写器可以将文档写入到输出流中 
            String fileName = "d:/" + title + ".doc";
            RtfWriter2.getInstance(doc, new FileOutputStream(fileName));  
            doc.open();  
            
            //粗体字
            Font boldFont = new Font();
            boldFont.setStyle(Font.BOLD);
            boldFont.setSize(16);
            
            //普通字体
            Font normalFont = new Font();
            boldFont.setSize(16);
            Paragraph p;
            
            for(int i = 0; i < jsonArray.size(); i++) {
            	JSONObject json = jsonArray.getJSONObject(i);
            	
            	//图片
            	String img = json.optString("img", "");
            	//粗体字
            	String bold = json.optString("bold", "");
            	//正常字体
            	String normal = json.optString("normal", "");
            	
            	//往doc里面插入内容
            	if(!StringUtils.equalsIgnoreCase("", img)) {
            		Image png = Image.getInstance(img);
                    doc.add(png);
            	}else if (!StringUtils.equalsIgnoreCase("", bold)) {
            		p = new Paragraph(bold);
                    p.setFont(boldFont);
                    doc.add(p);
            	}else if (!StringUtils.equalsIgnoreCase("", normal)) {
            		p = new Paragraph(normal); 
                    p.setFont(normalFont);
                    doc.add(p);
            	}
            }
            
            doc.close();  
        } catch (Exception e) {  
            e.printStackTrace();  
        }
	}
}
